# encoding:UTF-8

import tools.BaseConfig as baseConfig
import tools.getProxy as getProxy
import sys

from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
import time
from pyquery import PyQuery as pq

import tools.GlobalParam as gp

def getNewRank():
    periods = [{"id":"day","alias":"wx"},{"id":"week","alias":"wx"},{"id":"month","alias":"wx"},{"id":"weibo_day","alias":"wb"},{"id":"weibo_week","alias":"wb"},{"id":"toutiao_day","alias":"tt"},{"id":"pgcweek","alias":"pgc"},{"id":"routine_day","alias":"routine"},{"id":"routine_week","alias":"routine"},{"id":"routine_month","alias":"routine"}]
    for i, period in enumerate(periods):
        url = "http://www.newrank.cn/public/info/list.html?period=%s&type=data"
        driver=getAvailableDriver(url%period['id']);
        if(driver!=None):
            doBeginFetch(driver,period['alias'])
        else:
            print("无法连接对方网站，爬取失败")
            break;
        print("+++++++++++++++++++++++++++++++++++++++++++++++++++++++")
def doBeginFetch(driver,alias):
    print("begin")
    thirdClass = "-right-type-list"
    forthClass = "-right-type"
    doc = pq(driver.page_source)
    firstLvl = doc(".top-left").find("div")
    for i in firstLvl:
        if ("selected" in pq(i).attr("class")):
            print("first:",pq(i).text())
    forthLvl = doc("."+alias + forthClass).find("a")
    if (pq(forthLvl).html()==None):
        thirdClass = "-right-type-list-spe"
        forthLvl = doc("." + alias + forthClass+"-special").find("a")
    if(alias!="routine"):
        try:
            forthClick = driver.find_element_by_class_name(alias + thirdClass).find_elements_by_tag_name('a')
        except :
            if alias=="wx":
                forthClick = driver.find_element_by_class_name(alias + thirdClass + "-spe").find_elements_by_tag_name('a')
            else:
                forthClick = driver.find_element_by_class_name(alias + forthClass).find_elements_by_tag_name('a')
        for i,forth in enumerate(forthLvl):
            if(i!=0):
                # if (forth.get_attribute("data")):
                forthClick[i].click()
                time.sleep(10)
            getPageDetails(driver)
            print("====================================================")
    else:
        getPageDetails(driver)
        print("====================================================")

def getPageDetails(driver):
    print(readContent(pq(driver.page_source)))
    print("--------------------------------------------------")

def readContent(doc):
    contentList = []
    trs = doc("#table_list").find("tr")
    for tr in trs:
        singleList = []
        for td in pq(tr).find("td"):
            if pq(td).find(".copyright_title").attr("title") != None:
                singleList.append(pq(td).find(".copyright_title").attr("title"))
            else:
                singleList.append(pq(td).text())
        contentList.append(singleList)
    trs = doc("#more_list").find("tr")
    for tr in trs:
        singleList = []
        for td in pq(tr).find("td"):
            if pq(td).find(".copyright_title").attr("title") != None:
                singleList.append(pq(td).find(".copyright_title").attr("title"))
            else:
                singleList.append(pq(td).text())
        contentList.append(singleList)
    return contentList

def is_element_exist(driver,css):
    s = driver.find_elements_by_class_name(css)
    if len(s) == 0:
        print("元素未找到:%s"%css)
        return False
    else:
        return True

def getAvailableDriver(url):
    if (proxyListLen > gp.proxyNum):
        driver = baseConfig.getDriver(proxyList[gp.proxyNum])
        gp.proxyNum += 1
        try:
            driver.get(url)
            WebDriverWait(driver, 10).until(EC.title_contains(u"排名列表"))
        except:
            print("等待超时,进入下一个代理")
            print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
            driver.close()
            return getAvailableDriver(url)
        else:
            time.sleep(10)
            if (not is_element_exist(driver, "best")):
                print("爬取失败,进入下一个代理")
                print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
                driver.close()
                return getAvailableDriver(url)
            else:
                gp.proxyNum-=1
                return driver
    else:
        print("error")
        print(proxyListLen, gp.proxyNum)
        return None

if __name__ == "__main__":
    proxyList = getProxy.getProxyList()
    proxyListLen = len(proxyList)
    getNewRank()